import re
import jieba

# 数据清洗
# 初始数据item_profile.txt 每一行格式:商品名称,商品链接,价格,品牌,场景$主题$类别\n
# 处理后保存为item_profile1.txt 每一行格式: 商品名称（去除字母、数字和特别字符如‘/’）,品牌,场景$主题$类别\n

# 给出文档路径
# filename = "data/item_profile_set.txt"
# outfilename = "data/item_profile1.txt"
# 第二遍清洗
filename = "data/item_profile1.txt"
outfilename = "data/item_profile2.txt"
inputs = open(filename, 'r', encoding='UTF-8')
outputs = open(outfilename, 'w+', encoding='UTF-8')

''' 正则表达式匹配：各种括号及括号里的文字、数字、大小写字母以及一些符号 '''
# pattern = r'\(.*?\)|\（.*?\）|\【.*?\】|\“.*?\"|‘.*?\’|\s|[a-zA-Z0-9]|[\`\~\!\@\#\$\^\&\*\,
# \[\]\.\<\>\/\?\~\。\@\#\\\&\*\%\、\-\，\°\"\”\+|《|》]'
''' 但是并不能删除字母、数字，比如T恤、POLO衫、3日游 '''
pattern = r'\(.*?\)|\（.*?\）|\【.*?\】|\“.*?\“|\'.*?\’|\".*?\"|\<.*?\>|\[.*?\]|\#.*?\#]|\s|' \
          r'[\`\~\!\@\#\$\^\&\*\,\.\/\?\~\。\@\#\\\&\*\%\、\-\，\+\《\》\—\~\=\/\—\|] '
# 将输出结果写入item_profile1.txt中
for line in inputs:
    line1 = line.split(',', 4)
    # 删除游戏领域的商品
    check_scene = line1[-1].split('$')[0]
    if check_scene == '游戏':
        continue
    goods_name = line1[0]
    # 清洗商品名称
    line_seg = re.sub(pattern, "", goods_name)
    # print(line_seg)
    # # 去重
    # lines_seen = set()
    # if line_seg not in lines_seen:
    #     outputs.write(line_seg + ',' + line1[-2] + ',' + line1[-1])
    #     line_seg.add(goods_name)
    # else:
    #     continue
    outputs.write(line_seg + ',' + line1[-2] + ',' + line1[-1].strip('\n')+'\n')
    print("-----------正在清洗-----------")
outputs.close()
inputs.close()
print("清洗成功！！！")

# wordlist = []
# # with open("data/item_profile.txt", "r", encoding='utf8') as f:
# #     for line in f.readlines():
# #         line = line.strip('\t')
# #         data = line[0]
# #         print(data)
#
# w = jieba.lcut("爱因诗10本装不重复婴儿幼儿撕不烂早教卡书01", cut_all=True)
# for i in w:
#     wordlist.append(i)
#
#
# actree = ahocorasick.Automaton()
# for index, word in enumerate(wordlist):
#     actree.add_word(word, (index, word))
# actree.make_automaton()
#
# region_wds = []
# for i in actree.iter('【老货专卖】酱釉执壶'):
#     wd = i[1][1]
#     region_wds.append(wd)
#
# stop_wds = []
# for wd1 in region_wds:
#     for wd2 in region_wds:
#         if wd1 in wd2 and wd1 != wd2:
#             print(wd1, wd2)
#             stop_wds.append(wd1)
#
# final_wds = [i for i in region_wds if i not in stop_wds]
#
# print(region_wds)
# print(stop_wds)
# print(final_wds)
